package fileHandler;

import java.io.*;
import java.util.*;

/**
 * @author: ChenforCode
 * @date: 2018/12/29 12:00
 * @description: 将分词结果去重，并得到8份待分类的结果
 */
public class DuplicateRemove {
    public static void main(String[] args) {
        FileInputStream fis = null;
        InputStreamReader isr = null;
        BufferedReader br = null;
        Set<String> words = new HashSet<String>();
        try {
            String str;
            for (int i = 1; i <= 57; i++) {
                fis = new FileInputStream("E:\\LiuChen\\IDEA Java\\GAPoemGenerator\\corpus\\simplified\\jueju\\words\\" + "word" + i + ".json");
                isr = new InputStreamReader(fis);
                br = new BufferedReader(isr);
                while ((str = br.readLine()) != null) {
                    words.add(str);
                }
            }
        } catch (FileNotFoundException e) {
            System.out.println("找不到指定文件");
        } catch (IOException e) {
            System.out.println("读取文件失败");
        } finally {
            try {
                br.close();
                isr.close();
                fis.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        Iterator<String> it = words.iterator();
        List<String> words_list = new ArrayList<String>();
        while (it.hasNext()) {
            words_list.add(it.next());
        }


        FileOutputStream fos = null;
        OutputStreamWriter osw = null;
        BufferedWriter bw = null;
        int index = 0;
        try {
            for (int i = 0; i < 8; i++) {
                fos = new FileOutputStream(new File("E:\\LiuChen\\IDEA Java\\GAPoemGenerator\\corpus\\simplified\\jueju\\words_noduplicate\\" + "word" + i + ".json"));
                osw = new OutputStreamWriter(fos, "UTF-8");
                bw = new BufferedWriter(osw);
                if (i < 7) {
                    for (int j = 0; j < 660; j++) {
                        bw.write(words_list.get(index) + "\n");
                        index++;
                    }
                } else if (i == 7) {
                    for (int j = 0; j < 655; j++) {
                        bw.write(words_list.get(index) + "\n");
                        index++;
                    }
                }
                try {
                    bw.close();
                    osw.close();
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
